var http = require('https');
var fs = require('fs');
var cheerio = require('cheerio');
var request = require('request');
var xlsx = require('node-xlsx');

//初始url
var urls = ['https://www.youyoubai.com/index.php/2020/03/18/influxdb1-7%e7%89%88%e5%ae%89%e8%a3%85%e4%b8%8e%e9%85%8d%e7%bd%aewindows%e7%8e%af%e5%a2%83%e3%80%81jmeterinfluxdbgrafana%e6%80%a7%e8%83%bd%e7%9b%91%e6%8e%a7/','https://www.youyoubai.com/index.php/2019/11/22/jsr223%e6%8e%a7%e5%88%b6%e5%99%a8/']
var url = "";

for (var j = 0; j < urls.length; j++) {
    url = urls[j];
    startRequest(url);
}

function startRequest(x) {
    //采用http模块向服务器发起一次get请求      
    http.get(x, function (res) {
        var html = ''; //用来存储请求网页的整个html内容
        var detail = [];
        res.setEncoding('utf-8'); //防止中文乱码
        //监听data事件，每次取一块数据
        res.on('data', function (chunk) {
            html += chunk;
        });
        //监听end事件，如果整个网页内容的html都获取完毕，就执行回调函数
        res.on('end', function () {
            var obj = [] // 行
            var myxls = [] //xls文件 
            var table = {} // 文件里面的表
            var $ = cheerio.load(html); //采用cheerio模块解析html
            var data = [] // 列
            detail = $('html body main div div div div.container.pt-3 div.row article.article.col-lg-8 div.content').text().trim();
            data.push(detail)
            obj.push(data)
            table.data = obj
            table.name = 'Sheet'
            myxls.push(table)
            savedContent($, myxls);
        });
    }).on('error', function (err) {
        console.log(err);
    });
}

//在本地存储所爬取的学生内容
function savedContent($, obj) {
    var name = $('.main > .container .post-bar .meta.text-center > h1').text().trim();
    console.log(name);
    //转成buffer
    var file = xlsx.build(obj);
    fs.appendFile('./data/' + name + '.xlsx', file, 'binary', function (err) {
        if (err) {
            console.log(err);
        } else {
            console.log("爬取成功。");
        }
    });

}